#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jun 15 03:25:09 2020
将csv数据切分出训练和预测集
@author: ua17
"""
from process import *
from operator import itemgetter
from io import StringIO
from math import log2
import pandas as pd
import sys
import itertools



def read_mapper_output(file, separator = '\t'):
    for line in file:
        yield line.rstrip().split(separator)


# features = ["remain_seconds", "relative_direction", "relative_distance", "relative_x", "relative_y",
#             "abs_x", "abs_y", "block_start", "block_current","block_end", "sum_distance","dst_anchor",
#             "speed1", "anchor"]
out = "train_out.csv"
fw = open(out, 'w')
fw.write(",".join(columns3) + "\n")
fw.close()

out2 = "eval_out.csv"
fw2 = open(out2, 'w')
fw2.write(",".join(columns3) + "\n")
fw2.close()

#input_seqs = open("out-00002",'r')
input_seqs = sys.stdin
kv_sep = ','

print("start")


stdin_generator=read_mapper_output(input_seqs, kv_sep)
for key, values in itertools.groupby(stdin_generator,itemgetter(17)):
    if key == "loadingOrder":
        continue
    print(key, end="\t")
    all_v = '\n'.join([ ','.join(field_list) for field_list in values])
    data = pd.read_csv(StringIO(all_v), header=None)
    data.columns = columns3
    n_line = len(data)
    if data.iloc[0].dst_anchor == 1: #destination == 1
        print(" is a full journey ", end= "\t")
    data = data[data['complete_ratio'] < data.iloc[0].complete_ratio + 0.3]
    data2 = data[data['complete_ratio'] > data.iloc[-1].complete_ratio -0.1]

    n_sample = int(min(log2(len(data)) ** 2 * 1.2, len(data)))
    n_sample2 = int(min(log2(len(data2)) ** 1.5 , len(data2)))
    print(":{} => {} : {} ".format(n_line, n_sample, n_sample2))
    df_train = data.sample(n=n_sample)
    df_test = data2.sample(n=n_sample2)
   # df_train.sort_index(inplace=True)
    df_train.to_csv(out, index=False, header=False, mode='a', float_format="%.5f")
    df_test.to_csv(out2, index=False, header=False, mode='a', float_format="%.5f")